⚡ Pluto.jl ⚡

xxxxxxxxxx
 
begin
    using ReinforcementLearning
    using Flux
    using Statistics
    using Plots
    using StatsBase
end

47.7 s

Left Right Environment

5.5 μs

xxxxxxxxxx
 
begin
    Base.@kwdef mutable struct LeftRightEnv <: AbstractEnv
        reward::Float64 = 0.
        current_state::Int = 1
    end
​
    RLBase.state_space(env::LeftRightEnv) = Base.OneTo(2)
    RLBase.action_space(env::LeftRightEnv) = Base.OneTo(2)
​
    function (env::LeftRightEnv)(a::Int)
        if a == 2
            env.reward = 0.
            env.current_state = 2
        else
            s = sample(Weights([0.9, 0.1], 1.0))
            if s == 1
                env.reward = 0.
                env.current_state = 1
            else
                env.reward = 1.0
                env.current_state = 2
            end
        end
    end
​
    function RLBase.reset!(env::LeftRightEnv)
        env.current_state = 1
        env.reward = 0.
    end
​
    RLBase.reward(env::LeftRightEnv) = env.reward
    RLBase.is_terminated(env::LeftRightEnv) = env.current_state == 2
    RLBase.state(env::LeftRightEnv) = env.current_state
end

4.8 ms

world

# LeftRightEnv

## Traits

| Trait Type        |                                            Value |
|:----------------- | ------------------------------------------------:|
| NumAgentStyle     |          ReinforcementLearningBase.SingleAgent() |
| DynamicStyle      |           ReinforcementLearningBase.Sequential() |
| InformationStyle  | ReinforcementLearningBase.ImperfectInformation() |
| ChanceStyle       |           ReinforcementLearningBase.Stochastic() |
| RewardStyle       |           ReinforcementLearningBase.StepReward() |
| UtilityStyle      |           ReinforcementLearningBase.GeneralSum() |
| ActionStyle       |     ReinforcementLearningBase.MinimalActionSet() |
| StateStyle        |     ReinforcementLearningBase.Observation{Any}() |
| DefaultStateStyle |     ReinforcementLearningBase.Observation{Any}() |

## Is Environment Terminated?

No

## State Space

`Base.OneTo(2)`

## Action Space

`Base.OneTo(2)`

## Current State

```
1
```

xxxxxxxxxx
 
world = LeftRightEnv()

2.2 ms

xxxxxxxxxx
 
begin
    ns = length(state_space(world))
    na = length(action_space(world))
end 

81.0 ns

π_t

VBasedPolicy
├─ learner => MonteCarloLearner
│  ├─ approximator
│  │  ├─ 1
│  │  │  └─ TabularApproximator
│  │  │     ├─ table => 2-element Array{Float64,1}
│  │  │     └─ optimizer => Descent
│  │  │        └─ eta => 1.0
│  │  └─ 2
│  │     └─ TabularApproximator
│  │        ├─ table => 2-element Array{Float64,1}
│  │        └─ optimizer => InvDecay
│  │           ├─ gamma => 1.0
│  │           └─ state => IdDict
│  ├─ γ => 1.0
│  ├─ kind => ReinforcementLearningZoo.FirstVisit
│  └─ sampling => ReinforcementLearningZoo.OrdinaryImportanceSampling
└─ mapping => Main.var"#1#2"

xxxxxxxxxx
 
 π_t = VBasedPolicy(
    learner=MonteCarloLearner(
        approximator=(
            TabularVApproximator(;n_state=ns, opt=Descent(1.0)), # V
            TabularVApproximator(;n_state=ns, opt=InvDecay(1.0)) # Returns
            ),
        kind=FIRST_VISIT,
        sampling=ORDINARY_IMPORTANCE_SAMPLING,
        γ=1.0
        ),
    mapping= (env, V) -> 1
)

79.2 ms

xxxxxxxxxx
 
# A little ad-hoc here
RLBase.prob(::typeof(π_t), s, a) = a == 1 ? 1.0 : 0.

37.0 μs

xxxxxxxxxx
 
struct CollectValue <: AbstractHook
    values::Vector{Float64}
    CollectValue() = new([])
end

1.6 ms

xxxxxxxxxx
 
(f::CollectValue)(::PostEpisodeStage, agent, env) = push!(f.values, agent.policy.π_target.learner.approximator[2](1))

119 μs

xxxxxxxxxx
 
begin
    p = plot()
    for _ in 1:10
        fill!(π_t.learner.approximator[1].table, 0.)
        fill!(π_t.learner.approximator[2].table, 0.)
        empty!(π_t.learner.approximator[2].optimizer.state)
        agent = Agent(
            policy=OffPolicy(
                π_t,
                RandomPolicy(1:na)
                ),
            trajectory=VectorWSARTTrajectory()
        )
        hook = CollectValue()
        run(agent, world, StopAfterEpisode(100_000, is_show_progress=true),hook)
        plot!(p, hook.values, xscale = :log10)
    end
    p
end

14.0 s